{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "模型参数的访问、初始化和共享"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mxnet import init, nd\n",
    "from mxnet.gluon import nn\n",
    "\n",
    "net = nn.Sequential()\n",
    "net.add(nn.Dense(256, activation='relu'))\n",
    "net.add(nn.Dense(10))\n",
    "net.initialize()\n",
    "\n",
    "X = nd.random.uniform(shape=(2, 20))\n",
    "Y = net(X)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "询问模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(dense0_ (\n",
       "   Parameter dense0_weight (shape=(256, 20), dtype=float32)\n",
       "   Parameter dense0_bias (shape=(256,), dtype=float32)\n",
       " ), mxnet.gluon.parameter.ParameterDict)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].params, type(net[0].params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Parameter dense0_weight (shape=(256, 20), dtype=float32),\n",
       " Parameter dense0_weight (shape=(256, 20), dtype=float32))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].params['dense0_weight'], net[0].weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[ 0.06700657 -0.00369488  0.0418822  ... -0.05517294 -0.01194733\n",
       "  -0.00369594]\n",
       " [-0.03296221 -0.04391347  0.03839272 ...  0.05636378  0.02545484\n",
       "  -0.007007  ]\n",
       " [-0.0196689   0.01582889 -0.00881553 ...  0.01509629 -0.01908049\n",
       "  -0.02449339]\n",
       " ...\n",
       " [ 0.00010955  0.0439323  -0.04911506 ...  0.06975312  0.0449558\n",
       "  -0.03283203]\n",
       " [ 0.04106557  0.05671307 -0.00066976 ...  0.06387014 -0.01292654\n",
       "   0.00974177]\n",
       " [ 0.00297424 -0.0281784  -0.06881659 ... -0.04047417  0.00457048\n",
       "   0.05696651]]\n",
       "<NDArray 256x20 @cpu(0)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[0. 0. 0. ... 0. 0. 0.]\n",
       " [0. 0. 0. ... 0. 0. 0.]\n",
       " [0. 0. 0. ... 0. 0. 0.]\n",
       " ...\n",
       " [0. 0. 0. ... 0. 0. 0.]\n",
       " [0. 0. 0. ... 0. 0. 0.]\n",
       " [0. 0. 0. ... 0. 0. 0.]]\n",
       "<NDArray 256x20 @cpu(0)>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
       "<NDArray 10 @cpu(0)>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[1].bias.data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sequential0_ (\n",
       "  Parameter dense0_weight (shape=(256, 20), dtype=float32)\n",
       "  Parameter dense0_bias (shape=(256,), dtype=float32)\n",
       "  Parameter dense1_weight (shape=(10, 256), dtype=float32)\n",
       "  Parameter dense1_bias (shape=(10,), dtype=float32)\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.collect_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sequential0_ (\n",
       "  Parameter dense0_weight (shape=(256, 20), dtype=float32)\n",
       "  Parameter dense1_weight (shape=(10, 256), dtype=float32)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.collect_params('.*weight')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[ 0.00195949 -0.0173764   0.00047347  0.00145809  0.00326049  0.00457878\n",
       " -0.00894258  0.00493839 -0.00904343 -0.01214079  0.02156406  0.01093822\n",
       "  0.01827143 -0.0104467   0.01006219  0.0051742  -0.00806932  0.01376901\n",
       "  0.00205885  0.00994352]\n",
       "<NDArray 20 @cpu(0)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.initialize(init=init.Normal(sigma=0.01), force_reinit=True)\n",
    "net[0].weight.data()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n",
       "<NDArray 20 @cpu(0)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.initialize(init=init.Constant(1), force_reinit=True)\n",
    "net[0].weight.data()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[ 0.00512482 -0.06579044 -0.10849719 -0.09586414  0.06394844  0.06029618\n",
       " -0.03065033 -0.01086642  0.01929168  0.1003869  -0.09339568 -0.08703034\n",
       " -0.10472868 -0.09879824 -0.00352201 -0.11063069 -0.04257748  0.06548801\n",
       "  0.12987629 -0.13846186]\n",
       "<NDArray 20 @cpu(0)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.initialize(init=init.Xavier(), force_reinit=True)\n",
    "net[0].weight.data()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "自定义初始化法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init dense0_weight (256, 20)\n",
      "Init dense1_weight (10, 256)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "[-5.3659673  7.5773945  8.986376  -0.         8.827555   0.\n",
       "  5.9840508 -0.         0.         0.         7.4857597 -0.\n",
       " -0.         6.8910007  6.9788704 -6.1131554  0.         5.4665203\n",
       " -9.735263   9.485172 ]\n",
       "<NDArray 20 @cpu(0)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MyInit(init.Initializer):\n",
    "    def _init_weight(self, name, data):\n",
    "        print('Init', name, data.shape)\n",
    "        data[:] = nd.random.uniform(low=-10, high=10, shape=data.shape)\n",
    "        data *= data.abs() >= 5\n",
    "    \n",
    "net.initialize(MyInit(), force_reinit=True)\n",
    "net[0].weight.data()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[-4.3659673  8.5773945  9.986376   1.         9.827555   1.\n",
       "  6.9840508  1.         1.         1.         8.48576    1.\n",
       "  1.         7.8910007  7.9788704 -5.1131554  1.         6.4665203\n",
       " -8.735263  10.485172 ]\n",
       "<NDArray 20 @cpu(0)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.set_data(net[0].weight.data()+1)\n",
    "net[0].weight.data()[0]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "共享模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[1. 1. 1. 1. 1. 1. 1. 1.]\n",
       "<NDArray 8 @cpu(0)>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = nn.Sequential()\n",
    "shared = nn.Dense(8, activation='relu')\n",
    "net.add(nn.Dense(8, activation='relu'),\n",
    "       shared,\n",
    "       nn.Dense(8, activation='relu', params=shared.params),\n",
    "       nn.Dense(10))\n",
    "net.initialize()\n",
    "\n",
    "X = nd.random.uniform(shape=(2, 20))\n",
    "net(X)\n",
    "\n",
    "net[1].weight.data()[0] == net[2].weight.data()[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
